/*******************************************************************************
 *  SarsaLambdaAgent.h
 *  
 *  Created by Adam White, created on April 18 2007.
 *  Copyright (c) 2007 UAlberta. All rights reserved.
 *
 *  Description.
 *
 *		A Sarsa Lambda control Agent with Tile coding Function Approximation. 
 *		This agent has been tuned for the Acrobot task. It is based on 
 *		Sutton's code, found here:
 *
 *      http://www.cs.ualberta.ca/~sutton/MountainCar/MountainCar2.cpp
 *
 *      The code follows the psuedo-code for linear, gradient-descent 
 *		Sarsa(lambda) given in Figure 8.8 of the book "Reinforcement Learning: 
 *		An Introduction", by Sutton and Barto. One difference is that we use 
 *		the implementation trick mentioned on page 189 to only keep track of 
 *		the traces that are larger than "min-trace". 
 *					
 ********************************************************************************/
#ifndef SARSALAMMBDAAGENT_H
#define SARSALAMMBDAAGENT_H

#include "tiles.h"
#include "stdio.h"
#include "math.h"
#include <stdlib.h>


#if defined(linux) || defined(__linux)
#undef OS_WINDOWS
#else
#define OS_WINDOWS
#endif 
//#define OS_WINDOWS


#ifdef OS_WINDOWS
#include <windows.h>
double drand48();
#endif








#include <rlglue/Agent_common.h>
#include <rlglue/utils/C/RLStruct_util.h>
#include <rlglue/utils/C/TaskSpec_Parser.h>

//some constants
#define MEMORY_SIZE 10000                       //number of possible features 
#define	M_PI	3.141597

// RL glue agent methods 
void agent_init(const char* _ts); 
const action_t* agent_start(const observation_t* o); 
const action_t *agent_step(double reward, const observation_t *this_observation); 
void agent_end(double reward); 
void agent_cleanup(); 
const char* agent_message(const char* inMessage);



//Helper functions
void computeActionValues();                         
void computeActionValues(const action_t& a);                              
void computeActiveFeatures(const observation_t* o);                                    
int argmax(double* Q);            
void selectEpsilonGreedyAction(const observation_t* o, action_t& action);
void updateTraces();
void updateWeights(double delta);


//util function
template < typename T >
T **Allocate2DArray( int nRows, int nCols); 
template < typename T >
void Free2DArray(T** Array); 



// ------------------------ Suttons Trace Code -----------------------------------------------------
const int  MAX_NONZERO_TRACES=1000;


void ClearTrace(int f);                          // clear or zero-out trace, if any, for given feature
void ClearExistentTrace(int f, int loc);         // clear trace at given location in list of nonzero-traces

void DecayTraces(double decay_rate);              // decay all nonzero traces
void SetTrace(int f, double new_trace_value);     // set trace to given value
void IncreaseMinTrace();                         // increase minimal trace value, forcing more to 0, making room for new ones
// ------------------------ Suttons Trace Code -----------------------------------------------------
#endif